<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0,viewport-fit=cover"><title>深度学习代码笔记-01 | David 的博客</title><meta name="author" content="David"><meta name="copyright" content="David"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta name="description" content="深度学习代码笔记-011. 配置环境1.1 Conda 任选其一（推荐后者）  Anaconda 安装 Miniconda 安装   1.2 Conda 常用命令123456789101112131415161718192021222324252627282930# 显示所有环境conda env list# 显示当前环境下的包conda list# 创建conda环境conda create -">
<meta property="og:type" content="article">
<meta property="og:title" content="深度学习代码笔记-01">
<meta property="og:url" content="https://blog.david-deng.cn/2023/05/05/Deep-Learning-Code-Note/index.html">
<meta property="og:site_name" content="David 的博客">
<meta property="og:description" content="深度学习代码笔记-011. 配置环境1.1 Conda 任选其一（推荐后者）  Anaconda 安装 Miniconda 安装   1.2 Conda 常用命令123456789101112131415161718192021222324252627282930# 显示所有环境conda env list# 显示当前环境下的包conda list# 创建conda环境conda create -">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://jsd.012700.xyz/gh/jerryc127/CDN/img/material-4.png">
<meta property="article:published_time" content="2023-05-05T13:47:43.000Z">
<meta property="article:modified_time" content="2023-11-16T13:32:58.000Z">
<meta property="article:author" content="David">
<meta property="article:tag" content="笔记">
<meta property="article:tag" content="深度学习">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://jsd.012700.xyz/gh/jerryc127/CDN/img/material-4.png"><link rel="shortcut icon" href="/img/favicon.svg"><link rel="canonical" href="https://blog.david-deng.cn/2023/05/05/Deep-Learning-Code-Note/index.html"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free/css/all.min.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.css" media="print" onload="this.media='all'"><script>
    (() => {
      
    const saveToLocal = {
      set: (key, value, ttl) => {
        if (!ttl) return
        const expiry = Date.now() + ttl * 86400000
        localStorage.setItem(key, JSON.stringify({ value, expiry }))
      },
      get: key => {
        const itemStr = localStorage.getItem(key)
        if (!itemStr) return undefined
        const { value, expiry } = JSON.parse(itemStr)
        if (Date.now() > expiry) {
          localStorage.removeItem(key)
          return undefined
        }
        return value
      }
    }

    window.btf = {
      saveToLocal,
      getScript: (url, attr = {}) => new Promise((resolve, reject) => {
        const script = document.createElement('script')
        script.src = url
        script.async = true
        Object.entries(attr).forEach(([key, val]) => script.setAttribute(key, val))
        script.onload = script.onreadystatechange = () => {
          if (!script.readyState || /loaded|complete/.test(script.readyState)) resolve()
        }
        script.onerror = reject
        document.head.appendChild(script)
      }),
      getCSS: (url, id) => new Promise((resolve, reject) => {
        const link = document.createElement('link')
        link.rel = 'stylesheet'
        link.href = url
        if (id) link.id = id
        link.onload = link.onreadystatechange = () => {
          if (!link.readyState || /loaded|complete/.test(link.readyState)) resolve()
        }
        link.onerror = reject
        document.head.appendChild(link)
      }),
      addGlobalFn: (key, fn, name = false, parent = window) => {
        if (!false && key.startsWith('pjax')) return
        const globalFn = parent.globalFn || {}
        globalFn[key] = globalFn[key] || {}
        globalFn[key][name || Object.keys(globalFn[key]).length] = fn
        parent.globalFn = globalFn
      }
    }
  
      
      const activateDarkMode = () => {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      const activateLightMode = () => {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
        }
      }

      btf.activateDarkMode = activateDarkMode
      btf.activateLightMode = activateLightMode

      const theme = saveToLocal.get('theme')
    
          const mediaQueryDark = window.matchMedia('(prefers-color-scheme: dark)')
          const mediaQueryLight = window.matchMedia('(prefers-color-scheme: light)')
          
          if (theme === undefined) {
            if (mediaQueryLight.matches) activateLightMode()
            else if (mediaQueryDark.matches) activateDarkMode()
            else {
              const hour = new Date().getHours()
              const isNight = hour <= 6 || hour >= 18
              isNight ? activateDarkMode() : activateLightMode()
            }
            mediaQueryDark.addEventListener('change', () => {
              if (saveToLocal.get('theme') === undefined) {
                e.matches ? activateDarkMode() : activateLightMode()
              }
            })
          } else {
            theme === 'light' ? activateLightMode() : activateDarkMode()
          }
        
      
      const asideStatus = saveToLocal.get('aside-status')
      if (asideStatus !== undefined) {
        document.documentElement.classList.toggle('hide-aside', asideStatus === 'hide')
      }
    
      
    const detectApple = () => {
      if (/iPad|iPhone|iPod|Macintosh/.test(navigator.userAgent)) {
        document.documentElement.classList.add('apple')
      }
    }
    detectApple()
  
    })()
  </script><script>const GLOBAL_CONFIG = {
  root: '/',
  algolia: undefined,
  localSearch: undefined,
  translate: {"defaultEncoding":2,"translateDelay":0,"msgToTraditionalChinese":"繁","msgToSimplifiedChinese":"简"},
  highlight: {"plugin":"highlight.js","highlightCopy":true,"highlightLang":true,"highlightHeightLimit":false,"highlightFullpage":false,"highlightMacStyle":false},
  copy: {
    success: '复制成功',
    error: '复制失败',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: true,
    post: false
  },
  runtime: '',
  dateSuffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: {"limitCount":50,"languages":{"author":"作者: David","link":"链接: ","source":"来源: David 的博客","info":"著作权归作者所有。商业转载请联系作者获得授权，非商业转载请注明出处。"}},
  lightbox: 'null',
  Snackbar: {"chs_to_cht":"已切换为繁体中文","cht_to_chs":"已切换为简体中文","day_to_night":"已切换为深色模式","night_to_day":"已切换为浅色模式","bgLight":"#49b1f5","bgDark":"#121212","position":"bottom-left"},
  infinitegrid: {
    js: 'https://cdn.jsdelivr.net/npm/@egjs/infinitegrid/dist/infinitegrid.min.js',
    buttonText: '加载更多'
  },
  isPhotoFigcaption: false,
  islazyload: false,
  isAnchor: true,
  percent: {
    toc: true,
    rightside: false,
  },
  autoDarkmode: true
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = {
  title: '深度学习代码笔记-01',
  isPost: true,
  isHome: false,
  isHighlightShrink: false,
  isToc: true,
  isShuoshuo: false
}</script><style type="text/css">#toggle-sidebar {bottom: 80px}</style><meta name="generator" content="Hexo 7.3.0"></head><body><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="avatar-img text-center"><img src="/img/avatar.png" onerror="onerror=null;src='/img/loading.gif'" alt="avatar"/></div><div class="site-data text-center"><a href="/archives/"><div class="headline">文章</div><div class="length-num">27</div></a><a href="/tags/"><div class="headline">标签</div><div class="length-num">28</div></a><a href="/categories/"><div class="headline">分类</div><div class="length-num">28</div></a></div><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><span class="site-page group"><i class="fa-fw fas fa-compass"></i><span> 目录</span><i class="fas fa-chevron-down"></i></span><ul class="menus_item_child"><li><a class="site-page child" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 归档</span></a></li><li><a class="site-page child" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></li><li><a class="site-page child" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></li></ul></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友情链接</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div></div></div><div class="post" id="body-wrap"><header class="post-bg" id="page-header" style="background-image: url(https://jsd.012700.xyz/gh/jerryc127/CDN/img/material-4.png);"><nav id="nav"><span id="blog-info"><a class="nav-site-title" href="/"><span class="site-name">David 的博客</span></a><a class="nav-page-title" href="/"><span class="site-name">深度学习代码笔记-01</span></a></span><div id="menus"><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><span class="site-page group"><i class="fa-fw fas fa-compass"></i><span> 目录</span><i class="fas fa-chevron-down"></i></span><ul class="menus_item_child"><li><a class="site-page child" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 归档</span></a></li><li><a class="site-page child" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></li><li><a class="site-page child" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></li></ul></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友情链接</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div><div id="toggle-menu"><span class="site-page"><i class="fas fa-bars fa-fw"></i></span></div></div></nav><div id="post-info"><h1 class="post-title">深度学习代码笔记-01</h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="far fa-calendar-alt fa-fw post-meta-icon"></i><span class="post-meta-label">发表于</span><time class="post-meta-date-created" datetime="2023-05-05T13:47:43.000Z" title="发表于 2023-05-05 21:47:43">2023-05-05</time><span class="post-meta-separator">|</span><i class="fas fa-history fa-fw post-meta-icon"></i><span class="post-meta-label">更新于</span><time class="post-meta-date-updated" datetime="2023-11-16T13:32:58.000Z" title="更新于 2023-11-16 21:32:58">2023-11-16</time></span><span class="post-meta-categories"><span class="post-meta-separator">|</span><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/categories/%E7%AC%94%E8%AE%B0/">笔记</a><i class="fas fa-angle-right post-meta-separator"></i><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/categories/%E7%AC%94%E8%AE%B0/DL/">DL</a></span></div><div class="meta-secondline"><span class="post-meta-separator">|</span><span class="post-meta-wordcount"><i class="far fa-file-word fa-fw post-meta-icon"></i><span class="post-meta-label">总字数:</span><span class="word-count">4.9k</span><span class="post-meta-separator">|</span><i class="far fa-clock fa-fw post-meta-icon"></i><span class="post-meta-label">阅读时长:</span><span>21分钟</span></span><span class="post-meta-separator">|</span><span class="post-meta-pv-cv" id="" data-flag-title=""><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">浏览量:</span><span id="busuanzi_value_page_pv"><i class="fa-solid fa-spinner fa-spin"></i></span></span></div></div></div></header><main class="layout" id="content-inner"><div id="post"><article class="container post-content" id="article-container"><div id="post-outdate-notice" data="{&quot;limitDay&quot;:90,&quot;messagePrev&quot;:&quot;文章距离最近一次更新已经&quot;,&quot;messageNext&quot;:&quot;天，文章的内容可能已经过期。&quot;,&quot;postUpdate&quot;:&quot;2023-11-16 21:32:58&quot;}" hidden></div><h1 id="深度学习代码笔记-01"><a href="#深度学习代码笔记-01" class="headerlink" title="深度学习代码笔记-01"></a>深度学习代码笔记-01</h1><h2 id="1-配置环境"><a href="#1-配置环境" class="headerlink" title="1. 配置环境"></a>1. 配置环境</h2><h2 id="1-1-Conda"><a href="#1-1-Conda" class="headerlink" title="1.1 Conda"></a>1.1 <code>Conda</code></h2><blockquote>
<p>任选其一（推荐后者）</p>
<ol>
<li><a target="_blank" rel="noopener external nofollow noreferrer" href="https://repo.anaconda.com/archive/">Anaconda 安装</a></li>
<li><a target="_blank" rel="noopener external nofollow noreferrer" href="https://docs.conda.io/en/latest/miniconda.html">Miniconda 安装</a></li>
</ol>
</blockquote>
<h2 id="1-2-Conda-常用命令"><a href="#1-2-Conda-常用命令" class="headerlink" title="1.2 Conda 常用命令"></a>1.2 Conda 常用命令</h2><figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta prompt_"># </span><span class="language-bash">显示所有环境</span></span><br><span class="line">conda env list</span><br><span class="line"><span class="meta prompt_"></span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">显示当前环境下的包</span></span><br><span class="line">conda list</span><br><span class="line"><span class="meta prompt_"></span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">创建conda环境</span></span><br><span class="line">conda create -n 环境名 python=版本号</span><br><span class="line"><span class="meta prompt_"></span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">删除conda环境</span></span><br><span class="line">conda remove -n 环境名 --all</span><br><span class="line"><span class="meta prompt_"></span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">进入conda环境</span></span><br><span class="line">conda activate 环境名</span><br><span class="line"><span class="meta prompt_"></span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">退出conda环境</span></span><br><span class="line">conda deactivate</span><br><span class="line"><span class="meta prompt_"></span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">删除缓存</span></span><br><span class="line">conda clean -a -y</span><br><span class="line"><span class="meta prompt_"></span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">conda环境导出</span></span><br><span class="line">conda activate 环境名</span><br><span class="line">conda env export &gt; env.yaml</span><br><span class="line"><span class="meta prompt_"></span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">conda环境迁移</span></span><br><span class="line">conda env create -f env.yaml</span><br><span class="line"><span class="meta prompt_"></span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">conda国内源</span></span><br><span class="line">conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/</span><br></pre></td></tr></table></figure>

<h2 id="1-2-安装Pytorch"><a href="#1-2-安装Pytorch" class="headerlink" title="1.2 安装Pytorch"></a>1.2 安装<code>Pytorch</code></h2><p>官网地址：<a target="_blank" rel="noopener external nofollow noreferrer" href="https://pytorch.org/">PyTorch</a></p>
<p><img src="https://cdn.jsdelivr.net/gh/David-deng-01/images/blog/image-20230506004259724.png" alt="pythorch.png"></p>
<h2 id="1-分词任务"><a href="#1-分词任务" class="headerlink" title="1. 分词任务"></a>1. 分词任务</h2><h3 id="任务简介："><a href="#任务简介：" class="headerlink" title="任务简介："></a>任务简介：</h3><blockquote>
<p>模型内部是一系列的矩阵运算，只能处理数字。因此倘若需要让模型处理一个句子（比如判断这个句子是积极的，还是消极的），需要先把句子转为一串数字。所以在 NLP 学习中，我们需要先了解怎么将文本进行分词，并将每一个词都转化成对应的词向量。</p>
</blockquote>
<h3 id="任务步骤"><a href="#任务步骤" class="headerlink" title="任务步骤"></a>任务步骤</h3><ol>
<li>安装第三方库<code>pip install numpy nltk transformers</code></li>
<li>下载词向量文件 <code>glove.6B.50d.txt</code>, 下载地址：<a target="_blank" rel="noopener external nofollow noreferrer" href="https://www.kaggle.com/datasets/watts2/glove6b50dtxt">glove.6B.50d.txt</a></li>
<li>任务目标：将每个词转为词向量</li>
</ol>
<h3 id="Example-1-代码解释"><a href="#Example-1-代码解释" class="headerlink" title="Example 1 代码解释"></a>Example 1 代码解释</h3><ol>
<li><p>首先需要导入需要的第三方依赖</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 如果没有安装第三方依赖, 请安装</span></span><br><span class="line"><span class="comment"># pip install numpy nltk transformers</span></span><br><span class="line"><span class="keyword">from</span> typing <span class="keyword">import</span> <span class="type">Dict</span>, <span class="type">List</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">from</span> nltk <span class="keyword">import</span> word_tokenize</span><br><span class="line"><span class="keyword">from</span> numpy <span class="keyword">import</span> ndarray</span><br></pre></td></tr></table></figure>
</li>
<li><p>加载词向量文件</p>
<ul>
<li>从<code>glove.6B.50d.txt</code> 文件中按行读取词向量，每次读取一行</li>
<li>按照空格分割每一行的数据</li>
<li>分割得到的列表(list) 第一个元素是单词, 后面所有的元素是单词对应的词向量(vector)</li>
<li>将单词(word)作为 key, 词向量(vector) 作为 value, 存入 result 中</li>
</ul>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">read_glove_file</span>(<span class="params">self</span>) -&gt; <span class="type">Dict</span>[<span class="built_in">str</span>, <span class="type">List</span>[<span class="built_in">float</span>]]:</span><br><span class="line">    <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">        读取glove词向量文件, 并将其转换为字典形式</span></span><br><span class="line"><span class="string">        :return:  Dict[str, List[float]], key 为词, value 为词向量</span></span><br><span class="line"><span class="string">    &quot;&quot;&quot;</span></span><br><span class="line">    result: <span class="type">Dict</span>[<span class="built_in">str</span>, <span class="type">List</span>[<span class="built_in">float</span>]] = &#123;&#125;</span><br><span class="line">    glove_path = <span class="string">f&quot;<span class="subst">&#123;self.base_path&#125;</span>/<span class="subst">&#123;self.glove_file_name&#125;</span>&quot;</span></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">f&#x27;加载词向量文件：<span class="subst">&#123;glove_path&#125;</span>&#x27;</span>)</span><br><span class="line">    <span class="keyword">with</span> <span class="built_in">open</span>(glove_path, <span class="string">&#x27;r&#x27;</span>, encoding=<span class="string">&#x27;utf-8&#x27;</span>) <span class="keyword">as</span> file:</span><br><span class="line">        <span class="keyword">while</span> <span class="literal">True</span>:</span><br><span class="line">            line: <span class="built_in">str</span> = file.readline()  <span class="comment"># 读取一行</span></span><br><span class="line">            <span class="keyword">if</span> <span class="keyword">not</span> line:</span><br><span class="line">                <span class="keyword">break</span>  <span class="comment"># 如果没有读取成功</span></span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                line_split: <span class="built_in">list</span>[<span class="built_in">str</span>] = line.strip().split()  <span class="comment"># 按空格分割读取的一行数据</span></span><br><span class="line">                word: <span class="built_in">str</span> = line_split[<span class="number">0</span>]  <span class="comment"># 第一个为词，作为 key</span></span><br><span class="line">                vector: <span class="built_in">list</span>[<span class="built_in">float</span>] = <span class="built_in">list</span>(<span class="built_in">map</span>(<span class="built_in">float</span>, line_split[<span class="number">1</span>:]))  <span class="comment"># 除了第一个元素外, 其他元素组成对应的词向量</span></span><br><span class="line">                result[word] = vector  <span class="comment"># 将词作为 key, 向量作为 value, 存入结果中</span></span><br><span class="line">    <span class="keyword">return</span> result</span><br></pre></td></tr></table></figure>
</li>
<li><p>将句子转换成对应的词向量</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">run</span>(<span class="params">self</span>) -&gt; ndarray:</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">f&#x27;原始句子：<span class="subst">&#123;self.sentence&#125;</span>&#x27;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 第一步：分词</span></span><br><span class="line">    tokens: <span class="built_in">list</span>[<span class="built_in">str</span>] = word_tokenize(<span class="variable language_">self</span>.sentence)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">f&#x27;分词结果：<span class="subst">&#123;tokens&#125;</span>&#x27;</span>)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">f&#x27;序列长度：<span class="subst">&#123;<span class="built_in">len</span>(tokens)&#125;</span>&#x27;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 第二步：加载glove词向量文件, 提取每个word的词向量</span></span><br><span class="line">    word_to_vector_dict: <span class="type">Dict</span>[<span class="built_in">str</span>, <span class="type">List</span>[<span class="built_in">float</span>]] = <span class="variable language_">self</span>.read_glove_file()</span><br><span class="line">    dimension: <span class="built_in">int</span> = <span class="built_in">len</span>(word_to_vector_dict[<span class="string">&#x27;the&#x27;</span>])</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">f&#x27;词向量的大小：<span class="subst">&#123;<span class="built_in">len</span>(word_to_vector_dict)&#125;</span>&#x27;</span>)</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">f&quot;单词的维度：<span class="subst">&#123;dimension&#125;</span>&quot;</span>)</span><br><span class="line"></span><br><span class="line">    special_token_list: <span class="built_in">list</span>[<span class="built_in">str</span>] = [<span class="string">&#x27;unk&#x27;</span>, <span class="string">&#x27;pad&#x27;</span>, <span class="string">&#x27;cls&#x27;</span>, <span class="string">&#x27;sep&#x27;</span>]</span><br><span class="line">    <span class="keyword">for</span> sp_token <span class="keyword">in</span> special_token_list:</span><br><span class="line">        <span class="keyword">if</span> sp_token <span class="keyword">not</span> <span class="keyword">in</span> word_to_vector_dict:  <span class="comment"># 如果特殊字符不在词向量文件中</span></span><br><span class="line">            word_to_vector_dict[sp_token] = np.random.random(dimension).tolist()  <span class="comment"># 随机生成一些数字放入词向量文件中,作为特殊字符的词向量</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 第三步：转为词向量</span></span><br><span class="line">    arr = []</span><br><span class="line">    <span class="keyword">for</span> token <span class="keyword">in</span> tokens:</span><br><span class="line">        <span class="comment"># 将分词得到的 token 通过词向量表, 转换成对应的词向量</span></span><br><span class="line">        <span class="keyword">if</span> token <span class="keyword">not</span> <span class="keyword">in</span> word_to_vector_dict:</span><br><span class="line">            arr.append(word_to_vector_dict[<span class="string">&#x27;unk&#x27;</span>])</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            arr.append(word_to_vector_dict[token])</span><br><span class="line"></span><br><span class="line">    vector: ndarray = np.array(arr)  <span class="comment"># 将数组转成 numpy.ndarray</span></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">f&#x27;数组形状为：<span class="subst">&#123;vector.shape&#125;</span>&#x27;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 返回分词的结果</span></span><br><span class="line">    <span class="keyword">return</span> vector</span><br></pre></td></tr></table></figure>

<p><strong><code>操作步骤：</code></strong></p>
<ol>
<li>将句子(sentence)分词, 分成一个一个的单词(word)</li>
<li>将单词(word)通过 <code>word_to_vector_dict</code> 转换成对应的词向量 (vector)</li>
<li>将所有单词(word)的词向量(vector)按照顺序放入 <code>arr</code> 列表中，然后将 <code>arr</code> 数据类型转换成 <code>numpy.ndarray</code></li>
</ol>
</li>
<li><p>完整代码: <code>Example1.py</code></p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> typing <span class="keyword">import</span> <span class="type">Dict</span>, <span class="type">List</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> numpy <span class="keyword">as</span> np</span><br><span class="line"><span class="keyword">from</span> nltk <span class="keyword">import</span> word_tokenize</span><br><span class="line"><span class="keyword">from</span> numpy <span class="keyword">import</span> ndarray</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">Example1</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self</span>):</span><br><span class="line">        <span class="comment"># 原始的句子</span></span><br><span class="line">        <span class="variable language_">self</span>.sentence = <span class="string">&#x27;Commonsense knowledge and commonsense reasoning play &#x27;</span> \</span><br><span class="line">                        <span class="string">&#x27;a vital role in all aspects of machine intelligence,&#x27;</span> \</span><br><span class="line">                        <span class="string">&#x27;from language understanding to computer vision and &#x27;</span> \</span><br><span class="line">                        <span class="string">&#x27;robotics .&#x27;</span>.lower()</span><br><span class="line">        <span class="variable language_">self</span>.base_path = <span class="string">&quot;Model/glove&quot;</span>  <span class="comment"># 基础路径</span></span><br><span class="line">        <span class="variable language_">self</span>.glove_file_name = <span class="string">&quot;glove.6B.50d.txt&quot;</span>  <span class="comment"># 词向量文件名</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">run</span>(<span class="params">self</span>) -&gt; ndarray:</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">f&#x27;原始句子：<span class="subst">&#123;self.sentence&#125;</span>&#x27;</span>)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 第一步：分词</span></span><br><span class="line">        tokens: <span class="built_in">list</span>[<span class="built_in">str</span>] = word_tokenize(<span class="variable language_">self</span>.sentence)</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">f&#x27;分词结果：<span class="subst">&#123;tokens&#125;</span>&#x27;</span>)</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">f&#x27;序列长度：<span class="subst">&#123;<span class="built_in">len</span>(tokens)&#125;</span>&#x27;</span>)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 第二步：加载glove词向量文件, 提取每个word的词向量</span></span><br><span class="line">        word_to_vector_dict: <span class="type">Dict</span>[<span class="built_in">str</span>, <span class="type">List</span>[<span class="built_in">float</span>]] = <span class="variable language_">self</span>.read_glove_file()</span><br><span class="line">        dimension: <span class="built_in">int</span> = <span class="built_in">len</span>(word_to_vector_dict[<span class="string">&#x27;the&#x27;</span>])</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">f&#x27;词向量的大小：<span class="subst">&#123;<span class="built_in">len</span>(word_to_vector_dict)&#125;</span>&#x27;</span>)</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">f&quot;单词的维度：<span class="subst">&#123;dimension&#125;</span>&quot;</span>)</span><br><span class="line"></span><br><span class="line">        special_token_list: <span class="built_in">list</span>[<span class="built_in">str</span>] = [<span class="string">&#x27;unk&#x27;</span>, <span class="string">&#x27;pad&#x27;</span>, <span class="string">&#x27;cls&#x27;</span>, <span class="string">&#x27;sep&#x27;</span>]</span><br><span class="line">        <span class="keyword">for</span> sp_token <span class="keyword">in</span> special_token_list:</span><br><span class="line">            <span class="keyword">if</span> sp_token <span class="keyword">not</span> <span class="keyword">in</span> word_to_vector_dict:  <span class="comment"># 如果特殊字符不在词向量文件中</span></span><br><span class="line">                word_to_vector_dict[sp_token] = np.random.random(dimension).tolist()  <span class="comment"># 随机生成一些数字放入词向量文件中,作为特殊字符的词向量</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># 第三步：转为词向量</span></span><br><span class="line">        arr = []</span><br><span class="line">        <span class="keyword">for</span> token <span class="keyword">in</span> tokens:</span><br><span class="line">            <span class="comment"># 将分词得到的 token 通过词向量表, 转换成对应的词向量</span></span><br><span class="line">            <span class="keyword">if</span> token <span class="keyword">not</span> <span class="keyword">in</span> word_to_vector_dict:</span><br><span class="line">                arr.append(word_to_vector_dict[<span class="string">&#x27;unk&#x27;</span>])</span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                arr.append(word_to_vector_dict[token])</span><br><span class="line"></span><br><span class="line">        vector: ndarray = np.array(arr)  <span class="comment"># 将数组转成 numpy.ndarray</span></span><br><span class="line">        <span class="built_in">print</span>(<span class="string">f&#x27;数组形状为：<span class="subst">&#123;vector.shape&#125;</span>&#x27;</span>)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 返回分词的结果</span></span><br><span class="line">        <span class="keyword">return</span> vector</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">read_glove_file</span>(<span class="params">self</span>) -&gt; <span class="type">Dict</span>[<span class="built_in">str</span>, <span class="type">List</span>[<span class="built_in">float</span>]]:</span><br><span class="line">        <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">            读取glove词向量文件, 并将其转换为字典形式</span></span><br><span class="line"><span class="string">            :return:  Dict[str, List[float]], key 为词, value 为词向量</span></span><br><span class="line"><span class="string">        &quot;&quot;&quot;</span></span><br><span class="line">        result: <span class="type">Dict</span>[<span class="built_in">str</span>, <span class="type">List</span>[<span class="built_in">float</span>]] = &#123;&#125;</span><br><span class="line">        glove_path = <span class="string">f&quot;<span class="subst">&#123;self.base_path&#125;</span>/<span class="subst">&#123;self.glove_file_name&#125;</span>&quot;</span></span><br><span class="line">        <span class="built_in">print</span>(<span class="string">f&#x27;加载词向量文件：<span class="subst">&#123;glove_path&#125;</span>&#x27;</span>)</span><br><span class="line">        <span class="keyword">with</span> <span class="built_in">open</span>(glove_path, <span class="string">&#x27;r&#x27;</span>, encoding=<span class="string">&#x27;utf-8&#x27;</span>) <span class="keyword">as</span> file:</span><br><span class="line">            <span class="keyword">while</span> <span class="literal">True</span>:</span><br><span class="line">                line: <span class="built_in">str</span> = file.readline()  <span class="comment"># 读取一行</span></span><br><span class="line">                <span class="keyword">if</span> <span class="keyword">not</span> line:</span><br><span class="line">                    <span class="keyword">break</span>  <span class="comment"># 如果没有读取成功</span></span><br><span class="line">                <span class="keyword">else</span>:</span><br><span class="line">                    line_split: <span class="built_in">list</span>[<span class="built_in">str</span>] = line.strip().split()  <span class="comment"># 按空格分割读取的一行数据</span></span><br><span class="line">                    word: <span class="built_in">str</span> = line_split[<span class="number">0</span>]  <span class="comment"># 第一个为词，作为 key</span></span><br><span class="line">                    vector: <span class="built_in">list</span>[<span class="built_in">float</span>] = <span class="built_in">list</span>(<span class="built_in">map</span>(<span class="built_in">float</span>, line_split[<span class="number">1</span>:]))  <span class="comment"># 除了第一个元素外, 其他元素组成对应的词向量</span></span><br><span class="line">                    result[word] = vector  <span class="comment"># 将词作为 key, 向量作为 value, 存入结果中</span></span><br><span class="line">        <span class="keyword">return</span> result</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    v = Example1().run()</span><br><span class="line">    <span class="built_in">print</span>(v)</span><br><span class="line"></span><br></pre></td></tr></table></figure>




<p><code>代码运行结果：</code></p>
<img src="img/index/image-20230507233228291.png" alt="image-20230507233228291" style="zoom:100%;" /></li>
</ol>
<h3 id="Example2-代码解释"><a href="#Example2-代码解释" class="headerlink" title="Example2 代码解释"></a>Example2 代码解释</h3><ol>
<li><p><code>Example2.py</code> 和 <code>Example1.py </code>的区别：</p>
<ol>
<li><code>Example2.py</code> 是 <code>Example1.py</code> 的升级版，<code>Example1.py</code> 只是演示了如果将句子分词，将单词通过词向量文件转化成对应的词向量</li>
<li><code>Example2.py</code> 是我们在写模型中真正会用到的分词过程，不仅仅是将句子转换成对应的词向量</li>
<li>这两个文件最大的区别就是，在 <code>Example2.py</code> 中我们构建了自己的词汇表，然后将输入的句子通过词汇表转换成了对应的 token id</li>
</ol>
</li>
<li><p>首先需要先了解一下词汇表这个类 <code>Vocabulary</code></p>
<ol>
<li><p><code>Vocabulary</code> 类中有 5 个主要的成员变量，分别是: <code>id_to_word: dict[int, str]</code>、<code>word_to_id: dict[str, int]</code>、<code>word_feq: defaultdict[str, int]</code>、<code>special_token_list: list[str]</code>和<code>size: int</code></p>
</li>
<li><p>初始化方法 <code>def __init__(self)</code> 如下：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self</span>) -&gt; <span class="literal">None</span>:</span><br><span class="line">    <span class="variable language_">self</span>.word_to_id: <span class="type">Dict</span>[<span class="built_in">str</span>, <span class="built_in">int</span>] = &#123;&#125;  <span class="comment"># key: 单词, value: 单词 id</span></span><br><span class="line">    <span class="variable language_">self</span>.id_to_word: <span class="type">Dict</span>[<span class="built_in">int</span>, <span class="built_in">str</span>] = &#123;&#125;  <span class="comment"># key: 单词 id, value: 单词</span></span><br><span class="line">    <span class="variable language_">self</span>.word_feq: defaultdict[<span class="built_in">str</span>, <span class="built_in">int</span>] = defaultdict(<span class="built_in">int</span>)  <span class="comment"># 单词的频繁程度, 单词在词汇表中出现的次数</span></span><br><span class="line">    <span class="variable language_">self</span>.special_token_list: <span class="type">List</span>[<span class="built_in">str</span>] = []  <span class="comment"># 特殊的 token</span></span><br><span class="line">    <span class="variable language_">self</span>.size: <span class="built_in">int</span> = <span class="number">0</span>  <span class="comment"># 词汇表的大小, 初始时为 0</span></span><br><span class="line">    <span class="variable language_">self</span>.save_path = <span class="string">&quot;vocabulary&quot;</span>  <span class="comment"># 默认的文件保存路径</span></span><br><span class="line">    <span class="variable language_">self</span>.load_path = <span class="variable language_">self</span>.save_path  <span class="comment"># 默认的文件加载路径</span></span><br><span class="line">    <span class="variable language_">self</span>.keys = [<span class="string">&quot;word_to_id&quot;</span>, <span class="string">&quot;id_to_word&quot;</span>, <span class="string">&quot;special_token_list&quot;</span>, <span class="string">&quot;word_feq&quot;</span>,</span><br><span class="line">                 <span class="string">&quot;size&quot;</span>]  <span class="comment"># 保存时 json 文件的key, 也是 Vocabulary 的属性名称</span></span><br></pre></td></tr></table></figure>
</li>
<li><p><code>Vocabulary</code> 类中剩下的部分就是对这些变量的操作，详细的请看代码中的注释，具体代码如下：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 词汇表类</span></span><br><span class="line"><span class="keyword">import</span> json</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"><span class="keyword">from</span> collections <span class="keyword">import</span> defaultdict</span><br><span class="line"><span class="keyword">from</span> typing <span class="keyword">import</span> <span class="type">Dict</span>, <span class="type">List</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">Vocabulary</span>(<span class="title class_ inherited__">object</span>):</span><br><span class="line">    <span class="comment"># 成员变量的类型提示</span></span><br><span class="line">    id_to_word: <span class="built_in">dict</span>[<span class="built_in">int</span>, <span class="built_in">str</span>]</span><br><span class="line">    word_to_id: <span class="built_in">dict</span>[<span class="built_in">str</span>, <span class="built_in">int</span>]</span><br><span class="line">    word_feq: defaultdict[<span class="built_in">str</span>, <span class="built_in">int</span>]</span><br><span class="line">    special_token_list: <span class="built_in">list</span>[<span class="built_in">str</span>]</span><br><span class="line">    size: <span class="built_in">int</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self</span>) -&gt; <span class="literal">None</span>:</span><br><span class="line">        <span class="variable language_">self</span>.word_to_id: <span class="type">Dict</span>[<span class="built_in">str</span>, <span class="built_in">int</span>] = &#123;&#125;  <span class="comment"># key: 单词, value: 单词 id</span></span><br><span class="line">        <span class="variable language_">self</span>.id_to_word: <span class="type">Dict</span>[<span class="built_in">int</span>, <span class="built_in">str</span>] = &#123;&#125;  <span class="comment"># key: 单词 id, value: 单词</span></span><br><span class="line">        <span class="variable language_">self</span>.word_feq: defaultdict[<span class="built_in">str</span>, <span class="built_in">int</span>] = defaultdict(<span class="built_in">int</span>)  <span class="comment"># 单词的频繁程度, 单词在词汇表中出现的次数</span></span><br><span class="line">        <span class="variable language_">self</span>.special_token_list: <span class="type">List</span>[<span class="built_in">str</span>] = []  <span class="comment"># 特殊的 token</span></span><br><span class="line">        <span class="variable language_">self</span>.size: <span class="built_in">int</span> = <span class="number">0</span>  <span class="comment"># 词汇表的大小, 初始时为 0</span></span><br><span class="line">        <span class="variable language_">self</span>.save_path = <span class="string">&quot;vocabulary&quot;</span>  <span class="comment"># 默认的文件保存路径</span></span><br><span class="line">        <span class="variable language_">self</span>.load_path = <span class="variable language_">self</span>.save_path  <span class="comment"># 默认的文件加载路径</span></span><br><span class="line">        <span class="variable language_">self</span>.keys = [<span class="string">&quot;word_to_id&quot;</span>, <span class="string">&quot;id_to_word&quot;</span>, <span class="string">&quot;special_token_list&quot;</span>, <span class="string">&quot;word_feq&quot;</span>,</span><br><span class="line">                     <span class="string">&quot;size&quot;</span>]  <span class="comment"># 保存时 json 文件的key, 也是 Vocabulary 的属性名称</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">add_token</span>(<span class="params">self, token: <span class="built_in">str</span></span>) -&gt; <span class="built_in">int</span>:</span><br><span class="line">        <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">            将当前 token 添加到当前词汇表中</span></span><br><span class="line"><span class="string">            :param token: 待添加的 token</span></span><br><span class="line"><span class="string">            :return: 添加 token 后对应的 token id</span></span><br><span class="line"><span class="string">        &quot;&quot;&quot;</span></span><br><span class="line">        <span class="keyword">if</span> token <span class="keyword">not</span> <span class="keyword">in</span> <span class="variable language_">self</span>.word_to_id:</span><br><span class="line">            <span class="variable language_">self</span>.word_to_id[token] = <span class="variable language_">self</span>.size  <span class="comment"># 将 token 添加到 word_to_id 中</span></span><br><span class="line">            <span class="variable language_">self</span>.id_to_word[<span class="variable language_">self</span>.size] = token  <span class="comment"># 将 token 添加到 id_to_word 中</span></span><br><span class="line">            <span class="variable language_">self</span>.size += <span class="number">1</span>  <span class="comment"># 词汇表中 token 数量 +1</span></span><br><span class="line">        <span class="variable language_">self</span>.word_feq[token] += <span class="number">1</span>  <span class="comment"># 当前 token 出现的频率 +1</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.word_to_id[token]  <span class="comment"># 返回当前 token 的 id</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">add_tokens</span>(<span class="params">self, tokens: <span class="type">List</span>[<span class="built_in">str</span>]</span>) -&gt; <span class="type">List</span>[<span class="built_in">int</span>]:</span><br><span class="line">        <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">            批量添加 token 到词汇表中</span></span><br><span class="line"><span class="string">            :param tokens: 待添加的 token 列表</span></span><br><span class="line"><span class="string">            :return: 添加 token 后对应的 token id</span></span><br><span class="line"><span class="string">        &quot;&quot;&quot;</span></span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">list</span>(<span class="variable language_">self</span>.add_token(token) <span class="keyword">for</span> token <span class="keyword">in</span> tokens)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">add_special_token</span>(<span class="params">self, token: <span class="built_in">str</span></span>) -&gt; <span class="built_in">int</span>:</span><br><span class="line">        <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">            添加特殊的 token 或者说添加自定义的 token</span></span><br><span class="line"><span class="string">            :param token: 待添加的 token</span></span><br><span class="line"><span class="string">            :return: 添加 token 后对应的 token id</span></span><br><span class="line"><span class="string">        &quot;&quot;&quot;</span></span><br><span class="line">        <span class="keyword">if</span> token <span class="keyword">not</span> <span class="keyword">in</span> <span class="variable language_">self</span>.special_token_list:</span><br><span class="line">            <span class="comment"># 如果 token 没有在特殊 token 列表中出现</span></span><br><span class="line">            <span class="variable language_">self</span>.special_token_list.append(token)  <span class="comment"># 将当前 token 添加到特殊 token 列表中</span></span><br><span class="line">            <span class="keyword">return</span> <span class="variable language_">self</span>.add_token(token)  <span class="comment"># 将当前 token 添加到词汇表中, 并返回添加后的 token id</span></span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            <span class="comment"># 如果 token 已经在特殊 token 列表中出现过</span></span><br><span class="line">            <span class="keyword">return</span> <span class="variable language_">self</span>.word_to_id[token]  <span class="comment"># 查询词汇表, 返回对应的 token id</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">add_special_tokens</span>(<span class="params">self, tokens: <span class="type">List</span>[<span class="built_in">str</span>]</span>) -&gt; <span class="type">List</span>[<span class="built_in">int</span>]:</span><br><span class="line">        <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">            批量添加特殊的 token 到词汇表中</span></span><br><span class="line"><span class="string">            :param tokens: 待添加的 token 列表</span></span><br><span class="line"><span class="string">            :return: 添加 token 后对应的 token id</span></span><br><span class="line"><span class="string">        &quot;&quot;&quot;</span></span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">list</span>(<span class="variable language_">self</span>.add_special_token(token) <span class="keyword">for</span> token <span class="keyword">in</span> tokens)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">save_vocabulary_to_file</span>(<span class="params">self, path: <span class="built_in">str</span> = <span class="string">&quot;&quot;</span>, filename: <span class="built_in">str</span> = <span class="string">&quot;vocabulary.json&quot;</span></span>) -&gt; <span class="literal">None</span>:</span><br><span class="line">        <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">            保存 Vocabulary 到指定的路径下</span></span><br><span class="line"><span class="string">            :param filename: 默认文件名称s</span></span><br><span class="line"><span class="string">            :param path: 指定的保存路径</span></span><br><span class="line"><span class="string">            :return: None</span></span><br><span class="line"><span class="string">        &quot;&quot;&quot;</span></span><br><span class="line">        <span class="keyword">if</span> path == <span class="string">&quot;&quot;</span>:</span><br><span class="line">            path = <span class="variable language_">self</span>.save_path</span><br><span class="line"></span><br><span class="line">        os.makedirs(path, exist_ok=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line">        vocabulary_dict: <span class="built_in">dict</span> = &#123;key: <span class="built_in">getattr</span>(<span class="variable language_">self</span>, key) <span class="keyword">for</span> key <span class="keyword">in</span> <span class="variable language_">self</span>.keys&#125;  <span class="comment"># 通过属性名称获取对应的属性值</span></span><br><span class="line">        <span class="keyword">with</span> <span class="built_in">open</span>(<span class="string">f&quot;<span class="subst">&#123;path&#125;</span>/<span class="subst">&#123;filename&#125;</span>&quot;</span>, <span class="string">&quot;w&quot;</span>, encoding=<span class="string">&quot;utf-8&quot;</span>) <span class="keyword">as</span> file:</span><br><span class="line">            json.dump(vocabulary_dict, file)  <span class="comment"># 将 Vocabulary 类中指定的属性写入文件中</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">load_vocabulary_from_file</span>(<span class="params">self, path: <span class="built_in">str</span> = <span class="string">&quot;&quot;</span>, filename: <span class="built_in">str</span> = <span class="string">&quot;vocabulary.json&quot;</span></span>) -&gt; <span class="literal">None</span>:</span><br><span class="line">        <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">            读取指定路径下的 Vocabulary</span></span><br><span class="line"><span class="string">            :param filename: 默认文件名称</span></span><br><span class="line"><span class="string">            :param path:  指定的读取路径</span></span><br><span class="line"><span class="string">            :return: None</span></span><br><span class="line"><span class="string">        &quot;&quot;&quot;</span></span><br><span class="line">        <span class="keyword">if</span> path == <span class="string">&quot;&quot;</span>:</span><br><span class="line">            path = <span class="variable language_">self</span>.load_path</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> os.path.exists(path):</span><br><span class="line">            <span class="comment"># 文件路径不存在, 直接返回</span></span><br><span class="line">            <span class="built_in">print</span>(<span class="string">f&quot;【<span class="subst">&#123;path&#125;</span>/<span class="subst">&#123;filename&#125;</span>】, 文件不存在&quot;</span>)</span><br><span class="line">            <span class="keyword">return</span></span><br><span class="line"></span><br><span class="line">        <span class="keyword">with</span> <span class="built_in">open</span>(<span class="string">f&quot;<span class="subst">&#123;path&#125;</span>/<span class="subst">&#123;filename&#125;</span>&quot;</span>, <span class="string">&quot;r&quot;</span>, encoding=<span class="string">&quot;utf-8&quot;</span>) <span class="keyword">as</span> file:</span><br><span class="line">            vocabulary_dict: <span class="built_in">dict</span> = json.load(file)  <span class="comment"># 读取文件中的词汇表</span></span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> key <span class="keyword">in</span> <span class="variable language_">self</span>.keys:</span><br><span class="line">            <span class="built_in">setattr</span>(<span class="variable language_">self</span>, key, vocabulary_dict[key])  <span class="comment"># 通过 key 为 Vocabulary 的属性进行赋值</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">covert_token_to_id</span>(<span class="params">self, token: <span class="built_in">str</span></span>) -&gt; <span class="built_in">int</span>:</span><br><span class="line">        <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">            将 token 转换成 token id</span></span><br><span class="line"><span class="string">            :param token: 待转换的 token</span></span><br><span class="line"><span class="string">            :return: 指定 token 的 token id</span></span><br><span class="line"><span class="string">        &quot;&quot;&quot;</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.word_to_id[token]</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">convert_tokens_to_ids</span>(<span class="params">self, tokens: <span class="type">List</span>[<span class="built_in">str</span>]</span>) -&gt; <span class="type">List</span>[<span class="built_in">int</span>]:</span><br><span class="line">        <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">            批量 将 token 转换成 token id</span></span><br><span class="line"><span class="string">            :param tokens: 待转换的 token 列表</span></span><br><span class="line"><span class="string">            :return: token 列表中 token 的 token id</span></span><br><span class="line"><span class="string">        &quot;&quot;&quot;</span></span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">list</span>(<span class="variable language_">self</span>.covert_token_to_id(token) <span class="keyword">for</span> token <span class="keyword">in</span> tokens)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">convert_id_to_token</span>(<span class="params">self, token_id: <span class="built_in">int</span></span>) -&gt; <span class="built_in">str</span>:</span><br><span class="line">        <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">            通过 id 获取指定的 token</span></span><br><span class="line"><span class="string">            :param token_id: token id</span></span><br><span class="line"><span class="string">            :return: 指定的 token</span></span><br><span class="line"><span class="string">        &quot;&quot;&quot;</span></span><br><span class="line">        <span class="keyword">return</span> <span class="variable language_">self</span>.id_to_word[token_id]</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">convert_ids_to_tokens</span>(<span class="params">self, token_ids: <span class="type">List</span>[<span class="built_in">int</span>]</span>) -&gt; <span class="type">List</span>[<span class="built_in">str</span>]:</span><br><span class="line">        <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">            批量通过 id 获取指定的 token</span></span><br><span class="line"><span class="string">            :param token_ids: token id 列表</span></span><br><span class="line"><span class="string">            :return: 指定的 token 列表</span></span><br><span class="line"><span class="string">        &quot;&quot;&quot;</span></span><br><span class="line">        <span class="keyword">return</span> <span class="built_in">list</span>(<span class="variable language_">self</span>.convert_id_to_token(token_id) <span class="keyword">for</span> token_id <span class="keyword">in</span> token_ids)</span><br></pre></td></tr></table></figure></li>
</ol>
</li>
<li><p>首先我们需要根据自己的数据集创建对应的词汇表，这里演示使用的是 <code>Yelp</code> 数据集，详细的数据集可以在 <a target="_blank" rel="noopener external nofollow noreferrer" href="https://huggingface.co/">Hugging Face</a> 中下载，也可以使用我已经下载好的 <a target="_blank" rel="noopener external nofollow noreferrer" href="https://cdn.jsdelivr.net/gh/David-deng-01/images/dataset/yelp.jsonl">Yelp 数据集</a> 数据集里面的数据大致格式如下图所示：</p>
<img src="https://cdn.jsdelivr.net/gh/David-deng-01/images/blog/image-20230508172028197.png" alt="image-20230508172028197" style="zoom:100%;" />

<p><code>text</code> 表示句子， <code>label</code> 是句子的标签, 0 表示句子情感消极, 1 表示句子情感积极</p>
</li>
<li><p>根据数据集(dataset)创建词汇表(vocabulary)具体操作步骤如下:</p>
<ol>
<li>按行读取数据集，将句子取出</li>
<li>将句子进行分词，每个单词都存入词汇表中</li>
</ol>
<p>创建词汇表的代码如下：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">create_my_vocabulary</span>(<span class="params">data_file_path: <span class="built_in">str</span> = <span class="string">&quot;&quot;</span></span>) -&gt; Vocabulary:</span><br><span class="line">    <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">        根据自己的数据集, 创建自己的词汇表</span></span><br><span class="line"><span class="string">        :param data_file_path: 词汇表路径</span></span><br><span class="line"><span class="string">        :return:</span></span><br><span class="line"><span class="string">    &quot;&quot;&quot;</span></span><br><span class="line">    voca = Vocabulary()  <span class="comment"># 创建词汇表对象</span></span><br><span class="line">    <span class="keyword">if</span> data_file_path == <span class="string">&quot;&quot;</span>:</span><br><span class="line">        <span class="comment"># 如果没有传入数据集保存的位置, 则加载默认的词汇表</span></span><br><span class="line">        voca.load_vocabulary_from_file()</span><br><span class="line">        <span class="keyword">return</span> voca</span><br><span class="line"></span><br><span class="line">    voca.add_special_tokens([<span class="string">&#x27;unk&#x27;</span>, <span class="string">&#x27;pad&#x27;</span>, <span class="string">&#x27;cls&#x27;</span>, <span class="string">&#x27;sep&#x27;</span>])  <span class="comment"># 向词汇表中添加特殊 token</span></span><br><span class="line">    <span class="keyword">with</span> <span class="built_in">open</span>(data_file_path, <span class="string">&quot;r&quot;</span>, encoding=<span class="string">&quot;utf-8&quot;</span>) <span class="keyword">as</span> file:</span><br><span class="line">        <span class="keyword">for</span> line <span class="keyword">in</span> file.readlines():</span><br><span class="line">            text: <span class="built_in">str</span> = json.loads(line)[<span class="string">&#x27;text&#x27;</span>].lower()  <span class="comment"># 提取文件中的所有句子, 并将句子中的单词全部小写</span></span><br><span class="line">            voca.add_tokens(nltk.word_tokenize(text))  <span class="comment"># 使用分词工具, 将句子分词后添加到词汇表中</span></span><br><span class="line">    <span class="keyword">return</span> voca</span><br></pre></td></tr></table></figure>
</li>
<li><p>词汇表创建结束后，再将词向量文件加载到模型中，具体的加载操作与 <code>Example1.py</code> 中加载词向量文件的操作相似。具体代码如下：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">read_glove</span>(<span class="params">self</span>) -&gt; <span class="type">Dict</span>[<span class="built_in">str</span>, FloatTensor]:</span><br><span class="line">    <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">        读取预训练的词向量文件</span></span><br><span class="line"><span class="string">        :return: 单词和对应的词向量</span></span><br><span class="line"><span class="string">    &quot;&quot;&quot;</span></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;开始读取词向量文件&quot;</span>.center(<span class="number">50</span>, <span class="string">&quot;*&quot;</span>))</span><br><span class="line">    word_to_vector: <span class="type">Dict</span>[<span class="built_in">str</span>, FloatTensor] = &#123;&#125;</span><br><span class="line">    <span class="keyword">with</span> <span class="built_in">open</span>(<span class="variable language_">self</span>.glove_path, <span class="string">&quot;r&quot;</span>, encoding=<span class="string">&quot;utf-8&quot;</span>) <span class="keyword">as</span> file:</span><br><span class="line">        <span class="keyword">while</span> <span class="literal">True</span>:</span><br><span class="line">            line = file.readline()  <span class="comment"># 从词向量文件中读取一行数据</span></span><br><span class="line">            <span class="keyword">if</span> <span class="keyword">not</span> line:</span><br><span class="line">                <span class="keyword">break</span>  <span class="comment"># 如果没有读取到数据, 即 line is None, 跳出循环</span></span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                line_split: <span class="built_in">list</span>[<span class="built_in">str</span>] = line.strip().split()  <span class="comment"># 删除 line 前面和后面多余的空格, 并将 line 按照空格分开</span></span><br><span class="line">                word: <span class="built_in">str</span> = line_split[<span class="number">0</span>]  <span class="comment"># 单词, str</span></span><br><span class="line">                vector: FloatTensor = FloatTensor(<span class="built_in">list</span>(<span class="built_in">map</span>(<span class="built_in">float</span>, line_split[<span class="number">1</span>:])))  <span class="comment"># 单词对应的向量, FloatTensor</span></span><br><span class="line">                word_to_vector[word] = vector  <span class="comment"># key: word, value: vector</span></span><br><span class="line">    <span class="built_in">print</span>(<span class="string">&quot;词向量文件读取完成&quot;</span>.center(<span class="number">50</span>, <span class="string">&quot;*&quot;</span>))</span><br><span class="line">    <span class="keyword">return</span> word_to_vector</span><br></pre></td></tr></table></figure>
</li>
<li><p>接下来是创建模型，我们使用的是 <code>torch.nn.Embedding</code> 如果没有安装 <code>Pytorch</code> 请先安装，再进行接下来的操作</p>
</li>
<li><p>创建模型后，将一个批量(batch) 的数据放入模型中，在实际的神经网络训练中，我们也是一个批量一个批量的将数据送入模型中，而不是一条一条数据送入模型中</p>
</li>
<li><p>将数据输入模型后，在模型内部会进行填充(padding)操作，这是因为模型一般都是对矩阵进行操作，但是我们输入的句子可能有长有短，所以需要将短的句子使用特殊的 token 填充到和当前 batch 中最长的句子一样长，这样模型才能进行处理</p>
</li>
<li><p><code>Example2.py</code> 完整代码如下：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> json</span><br><span class="line"><span class="keyword">import</span> os</span><br><span class="line"><span class="keyword">from</span> typing <span class="keyword">import</span> <span class="type">List</span>, <span class="type">Dict</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> nltk</span><br><span class="line"><span class="keyword">import</span> torch</span><br><span class="line"><span class="keyword">from</span> torch <span class="keyword">import</span> nn <span class="keyword">as</span> nn, FloatTensor, LongTensor</span><br><span class="line"><span class="keyword">from</span> torch.nn <span class="keyword">import</span> Embedding</span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> utils.Vocabulary <span class="keyword">import</span> Vocabulary</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">create_my_vocabulary</span>(<span class="params">data_file_path: <span class="built_in">str</span> = <span class="string">&quot;&quot;</span></span>) -&gt; Vocabulary:</span><br><span class="line">    <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">        根据自己的数据集, 创建自己的词汇表</span></span><br><span class="line"><span class="string">        :param data_file_path: 词汇表路径</span></span><br><span class="line"><span class="string">        :return:</span></span><br><span class="line"><span class="string">    &quot;&quot;&quot;</span></span><br><span class="line">    voca = Vocabulary()  <span class="comment"># 创建词汇表对象</span></span><br><span class="line">    <span class="keyword">if</span> data_file_path == <span class="string">&quot;&quot;</span>:</span><br><span class="line">        <span class="comment"># 如果没有传入数据集保存的位置, 则加载默认的词汇表</span></span><br><span class="line">        voca.load_vocabulary_from_file()</span><br><span class="line">        <span class="keyword">return</span> voca</span><br><span class="line"></span><br><span class="line">    voca.add_special_tokens([<span class="string">&#x27;unk&#x27;</span>, <span class="string">&#x27;pad&#x27;</span>, <span class="string">&#x27;cls&#x27;</span>, <span class="string">&#x27;sep&#x27;</span>])  <span class="comment"># 向词汇表中添加特殊 token</span></span><br><span class="line">    <span class="keyword">with</span> <span class="built_in">open</span>(data_file_path, <span class="string">&quot;r&quot;</span>, encoding=<span class="string">&quot;utf-8&quot;</span>) <span class="keyword">as</span> file:</span><br><span class="line">        <span class="keyword">for</span> line <span class="keyword">in</span> file.readlines():</span><br><span class="line">            text: <span class="built_in">str</span> = json.loads(line)[<span class="string">&#x27;text&#x27;</span>].lower()  <span class="comment"># 提取文件中的所有句子, 并将句子中的单词全部小写</span></span><br><span class="line">            voca.add_tokens(nltk.word_tokenize(text))  <span class="comment"># 使用分词工具, 将句子分词后添加到词汇表中</span></span><br><span class="line">    <span class="keyword">return</span> voca</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="title class_">Example2</span>(nn.Module):</span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">__init__</span>(<span class="params">self, vocabulary: Vocabulary = <span class="literal">None</span>, dimension: <span class="built_in">int</span> = <span class="number">50</span></span>):</span><br><span class="line">        <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">            初始化方法</span></span><br><span class="line"><span class="string">            :param vocabulary: 词汇表</span></span><br><span class="line"><span class="string">            :param dimension: 每个单词的维度, 默认为 50 维</span></span><br><span class="line"><span class="string">        &quot;&quot;&quot;</span></span><br><span class="line">        <span class="comment"># 调用父类 nn.Module 的初始化方法</span></span><br><span class="line">        <span class="built_in">super</span>(Example2, <span class="variable language_">self</span>).__init__()</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 保存词汇表</span></span><br><span class="line">        <span class="variable language_">self</span>.vocabulary: Vocabulary = vocabulary</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 创建一个嵌入层 embedding, 是一个二维矩阵, 形状为 (word_number, dimension)</span></span><br><span class="line">        <span class="variable language_">self</span>.embedding: Embedding = nn.Embedding(num_embeddings=vocabulary.size, embedding_dim=dimension)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 路径</span></span><br><span class="line">        <span class="variable language_">self</span>.glove_path = <span class="string">&quot;Model/glove/glove.6B.50d.txt&quot;</span>  <span class="comment"># 预训练的词向量文件位置</span></span><br><span class="line">        <span class="variable language_">self</span>.output_dir = <span class="string">&quot;output/example2&quot;</span>  <span class="comment"># 输出文件保存位置</span></span><br><span class="line">        <span class="variable language_">self</span>.vocabulary_filename = <span class="string">&quot;vocabulary.json&quot;</span>  <span class="comment"># 输出的词汇表文件名称</span></span><br><span class="line">        <span class="variable language_">self</span>.vocabulary_ckpt_path = <span class="string">f&quot;<span class="subst">&#123;self.output_dir&#125;</span>/<span class="subst">&#123;self.vocabulary_filename&#125;</span>&quot;</span>  <span class="comment"># 词汇表保存位置</span></span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">init_embedding</span>(<span class="params">self</span>) -&gt; <span class="literal">None</span>:</span><br><span class="line">        <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">            初始化</span></span><br><span class="line"><span class="string">            :return: None</span></span><br><span class="line"><span class="string">        &quot;&quot;&quot;</span></span><br><span class="line">        <span class="comment"># 1.加载预训练的词向量文件</span></span><br><span class="line">        word_to_vector = <span class="variable language_">self</span>.read_glove()</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 2. 计算词向量文件相对于自己的数据集的命中率</span></span><br><span class="line">        hit = <span class="number">0</span>  <span class="comment"># 命中次数</span></span><br><span class="line">        unhit_token = []  <span class="comment"># 未命中的 token</span></span><br><span class="line">        <span class="keyword">for</span> word, word_id <span class="keyword">in</span> <span class="variable language_">self</span>.vocabulary.word_to_id.items():</span><br><span class="line">            <span class="keyword">if</span> word <span class="keyword">in</span> word_to_vector:</span><br><span class="line">                hit += <span class="number">1</span>  <span class="comment"># 命中的次数 +1</span></span><br><span class="line">                vector = word_to_vector[word]  <span class="comment"># 获取当前 word 对应的 vector</span></span><br><span class="line">                <span class="variable language_">self</span>.embedding.weight.data[word_id] = vector  <span class="comment"># 给词向量赋值</span></span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                unhit_token.append(word)</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">f&quot;由数据库创建的单词表的大小为: <span class="subst">&#123;self.vocabulary.size&#125;</span>&quot;</span>)</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">f&quot;其中<span class="subst">&#123;hit&#125;</span>个词有预训练的词向量, 命中率为: <span class="subst">&#123;hit / self.vocabulary.size&#125;</span>&quot;</span>)</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">f&quot;没有预训练词向量的词有: <span class="subst">&#123;unhit_token&#125;</span>, 它们的词向量是随机初始化的&quot;</span>)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">read_glove</span>(<span class="params">self</span>) -&gt; <span class="type">Dict</span>[<span class="built_in">str</span>, FloatTensor]:</span><br><span class="line">        <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">            读取预训练的词向量文件</span></span><br><span class="line"><span class="string">            :return: 单词和对应的词向量</span></span><br><span class="line"><span class="string">        &quot;&quot;&quot;</span></span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;开始读取词向量文件&quot;</span>.center(<span class="number">50</span>, <span class="string">&quot;*&quot;</span>))</span><br><span class="line">        word_to_vector: <span class="type">Dict</span>[<span class="built_in">str</span>, FloatTensor] = &#123;&#125;</span><br><span class="line">        <span class="keyword">with</span> <span class="built_in">open</span>(<span class="variable language_">self</span>.glove_path, <span class="string">&quot;r&quot;</span>, encoding=<span class="string">&quot;utf-8&quot;</span>) <span class="keyword">as</span> file:</span><br><span class="line">            <span class="keyword">while</span> <span class="literal">True</span>:</span><br><span class="line">                line = file.readline()  <span class="comment"># 从词向量文件中读取一行数据</span></span><br><span class="line">                <span class="keyword">if</span> <span class="keyword">not</span> line:</span><br><span class="line">                    <span class="keyword">break</span>  <span class="comment"># 如果没有读取到数据, 即 line is None, 跳出循环</span></span><br><span class="line">                <span class="keyword">else</span>:</span><br><span class="line">                    line_split: <span class="built_in">list</span>[<span class="built_in">str</span>] = line.strip().split()  <span class="comment"># 删除 line 前面和后面多余的空格, 并将 line 按照空格分开</span></span><br><span class="line">                    word: <span class="built_in">str</span> = line_split[<span class="number">0</span>]  <span class="comment"># 单词, str</span></span><br><span class="line">                    vector: FloatTensor = FloatTensor(<span class="built_in">list</span>(<span class="built_in">map</span>(<span class="built_in">float</span>, line_split[<span class="number">1</span>:])))  <span class="comment"># 单词对应的向量, FloatTensor</span></span><br><span class="line">                    word_to_vector[word] = vector  <span class="comment"># key: word, value: vector</span></span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&quot;词向量文件读取完成&quot;</span>.center(<span class="number">50</span>, <span class="string">&quot;*&quot;</span>))</span><br><span class="line">        <span class="keyword">return</span> word_to_vector</span><br><span class="line"></span><br><span class="line">    <span class="keyword">def</span> <span class="title function_">forward</span>(<span class="params">self, input_text: <span class="type">List</span>[<span class="built_in">str</span>]</span>) -&gt; FloatTensor:</span><br><span class="line">        <span class="string">&quot;&quot;&quot;</span></span><br><span class="line"><span class="string">            查询 input_text 中的句子的</span></span><br><span class="line"><span class="string">            :param input_text: 一个个句子组成的列表</span></span><br><span class="line"><span class="string">            :return:</span></span><br><span class="line"><span class="string">        &quot;&quot;&quot;</span></span><br><span class="line">        <span class="comment"># 1. 将句子全部进行分词转为相应 token id</span></span><br><span class="line">        word_list: <span class="built_in">list</span>[<span class="built_in">list</span>[<span class="built_in">str</span>]] = <span class="built_in">list</span>(nltk.word_tokenize(sentence) <span class="keyword">for</span> sentence <span class="keyword">in</span> input_text)</span><br><span class="line">        input_ids: <span class="built_in">list</span>[<span class="built_in">list</span>[<span class="built_in">int</span>]] = <span class="built_in">list</span>(<span class="variable language_">self</span>.vocabulary.convert_tokens_to_ids(sequence) <span class="keyword">for</span> sequence <span class="keyword">in</span> word_list)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 2. 计算序列的最大长度, 方便以后的 padding 操作, 因为模型输入的是矩阵, 如果句子的长度不一样, 我们应该进行 padding 操作</span></span><br><span class="line">        max_len: <span class="built_in">int</span> = <span class="built_in">max</span>(<span class="built_in">len</span>(sequence) <span class="keyword">for</span> sequence <span class="keyword">in</span> input_ids)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 3. 获取用于 padding 的 token id, 即 &quot;pad&quot; 的 id</span></span><br><span class="line">        pad_id: <span class="built_in">int</span> = <span class="variable language_">self</span>.vocabulary.word_to_id[<span class="string">&quot;pad&quot;</span>]</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 4. 因为 input_ids 中 list 的长度不一, 所以需要统一长度, 即 padding 操作</span></span><br><span class="line">        input_ids: <span class="built_in">list</span>[<span class="built_in">list</span>[<span class="built_in">int</span>]] = <span class="built_in">list</span>(sequence + [pad_id] * (max_len - <span class="built_in">len</span>(sequence)) <span class="keyword">for</span> sequence <span class="keyword">in</span> input_ids)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 5. 将 input_ids 转成 tensor, shape &gt;&gt; [bath_size, max_len]</span></span><br><span class="line">        input_ids: LongTensor = LongTensor(input_ids)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># 6. 将 input_ids 转成词向量</span></span><br><span class="line">        input_embedding: FloatTensor = <span class="variable language_">self</span>.embedding(input_ids)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">return</span> input_embedding</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">run</span>(<span class="params">dimension: <span class="built_in">int</span> = <span class="number">50</span></span>):</span><br><span class="line">    data_file_path = <span class="string">&quot;data/yelp.jsonl&quot;</span>  <span class="comment"># 自己的数据集的位置</span></span><br><span class="line">    output_dir = <span class="string">&quot;output/example2&quot;</span>  <span class="comment"># 输出文件保存位置</span></span><br><span class="line">    embedding_layer_ckpt_path = <span class="string">f&quot;<span class="subst">&#123;output_dir&#125;</span>/embedding_layer.pt&quot;</span>  <span class="comment"># 嵌入层保存位置</span></span><br><span class="line"></span><br><span class="line">    <span class="comment"># 1. 创建输出文件目录, 如果不存在的话</span></span><br><span class="line">    os.makedirs(output_dir, exist_ok=<span class="literal">True</span>)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 2. 根据自己的数据集创建词汇表</span></span><br><span class="line">    voca = create_my_vocabulary(data_file_path)</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 3. 创建 embedding 层</span></span><br><span class="line">    embedding_layer = Example2(vocabulary=voca, dimension=dimension)</span><br><span class="line">    <span class="keyword">if</span> embedding_layer_ckpt_path <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">        embedding_layer.init_embedding()</span><br><span class="line">        torch.save(embedding_layer.state_dict(), <span class="string">f&quot;<span class="subst">&#123;output_dir&#125;</span>/embedding_layer.pt&quot;</span>)</span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        embedding_layer.load_state_dict(torch.load(embedding_layer_ckpt_path))</span><br><span class="line"></span><br><span class="line">    <span class="comment"># 4. 将一个 batch 的 sentences 转为向量</span></span><br><span class="line">    <span class="comment"># 演示的batch size = 4</span></span><br><span class="line">    batch_text = [</span><br><span class="line">        <span class="string">&quot;ever since joes has changed hands it &#x27;s just gotten worse and worse .&quot;</span>,</span><br><span class="line">        <span class="string">&quot;there is definitely not enough room in that part of the venue .&quot;</span>,</span><br><span class="line">        <span class="string">&quot;so basically tasted watered down .&quot;</span>,</span><br><span class="line">        <span class="string">&quot;she said she &#x27;d be back and disappeared for a few minutes .&quot;</span></span><br><span class="line">    ]</span><br><span class="line">    <span class="built_in">print</span>([item.shape <span class="keyword">for</span> item <span class="keyword">in</span> embedding_layer.forward(batch_text)])</span><br><span class="line">    <span class="built_in">print</span>(embedding_layer(batch_text).shape)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    run(dimension=<span class="number">50</span>)</span><br><span class="line"></span><br></pre></td></tr></table></figure>


<p><code>代码运行结果如下</code>：</p>
</li>
</ol>
<img src="https://cdn.jsdelivr.net/gh/David-deng-01/images/blog/image-20230508180244621.png" alt="image-20230508180244621" style="zoom:100%;" />

<h2 id="2-句子情感分类任务"><a href="#2-句子情感分类任务" class="headerlink" title="2. 句子情感分类任务"></a>2. 句子情感分类任务</h2><h2 id="3-对话生成任务"><a href="#3-对话生成任务" class="headerlink" title="3. 对话生成任务"></a>3. 对话生成任务</h2></article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta"><i class="fas fa-circle-user fa-fw"></i>文章作者: </span><span class="post-copyright-info"><a href="https://blog.david-deng.cn">David</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta"><i class="fas fa-square-arrow-up-right fa-fw"></i>文章链接: </span><span class="post-copyright-info"><a href="https://blog.david-deng.cn/2023/05/05/Deep-Learning-Code-Note/">https://blog.david-deng.cn/2023/05/05/Deep-Learning-Code-Note/</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta"><i class="fas fa-circle-exclamation fa-fw"></i>版权声明: </span><span class="post-copyright-info">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" rel="external nofollow noreferrer" target="_blank">CC BY-NC-SA 4.0</a> 许可协议。转载请注明来源 <a href="https://blog.david-deng.cn" target="_blank">David 的博客</a>！</span></div></div><div class="tag_share"><div class="post-meta__tag-list"><a class="post-meta__tags" href="/tags/%E7%AC%94%E8%AE%B0/">笔记</a><a class="post-meta__tags" href="/tags/%E6%B7%B1%E5%BA%A6%E5%AD%A6%E4%B9%A0/">深度学习</a></div><div class="post-share"><div class="social-share" data-image="https://jsd.012700.xyz/gh/jerryc127/CDN/img/material-4.png" data-sites="facebook,twitter,wechat,weibo,qq"></div><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/butterfly-extsrc/sharejs/dist/css/share.min.css" media="print" onload="this.media='all'"><script src="https://cdn.jsdelivr.net/npm/butterfly-extsrc/sharejs/dist/js/social-share.min.js" defer></script></div></div><div class="post-reward"><div class="reward-button"><i class="fas fa-qrcode"></i>赞助</div><div class="reward-main"><ul class="reward-all"><li class="reward-item"><a href="https://cdn.jsdelivr.net/gh/David-deng-01/images/blog/WeChatPay.jpg" rel="external nofollow noreferrer" target="_blank"><img class="post-qr-code-img" src="https://cdn.jsdelivr.net/gh/David-deng-01/images/blog/WeChatPay.jpg" alt="微信"/></a><div class="post-qr-code-desc">微信</div></li><li class="reward-item"><a href="https://cdn.jsdelivr.net/gh/David-deng-01/images/blog/Alipay.jpg" rel="external nofollow noreferrer" target="_blank"><img class="post-qr-code-img" src="https://cdn.jsdelivr.net/gh/David-deng-01/images/blog/Alipay.jpg" alt="支付宝"/></a><div class="post-qr-code-desc">支付宝</div></li></ul></div></div><nav class="pagination-post" id="pagination"><a class="pagination-related" href="/2023/05/04/educoder-ml-homework/" title="头歌 - 机器学习 - 决策树"><img class="cover" src="https://jsd.012700.xyz/gh/jerryc127/CDN/img/material-2.png" onerror="onerror=null;src='/img/404.jpg'" alt="cover of previous post"><div class="info"><div class="info-1"><div class="info-item-1">上一篇</div><div class="info-item-2">头歌 - 机器学习 - 决策树</div></div><div class="info-2"><div class="info-item-1">【educoder】 机器学习 --- 决策树第 1...</div></div></div></a><a class="pagination-related" href="/2023/05/06/typora-shortcut-keys/" title="typora 快捷键"><img class="cover" src="https://jsd.012700.xyz/gh/jerryc127/CDN/img/material-9.png" onerror="onerror=null;src='/img/404.jpg'" alt="cover of next post"><div class="info text-right"><div class="info-1"><div class="info-item-1">下一篇</div><div class="info-item-2">typora 快捷键</div></div><div class="info-2"><div class="info-item-1">Shortcut Keys摘要您可以使用快捷键快速插入或修改样式，或执行 Typora 支持的其他操作。您可以在菜单栏中的每个菜单项的右侧找到快捷键。 常用快捷键自动补全在 macOS 上，您可以按 ESC 键打开内联数学的内联预览，以及表情符号的自动完成等。 文件相关   功能 快捷键 (Windows&#x2F;Linux) 快捷键 (macOS)    New Ctrl + N Command + N   New Window Ctrl + Shift + N Command +Shift + N   New Tab (Not Supported) Command + T   Open Ctrl + O Command + O   Open Quickly Ctrl + P Command + Shift + O   Reopen Closed File Ctrl + Shift + T Command + Shift + T   Save Ctrl + S Command + S   Save As &#x2F; Duplicate Ctrl + Shift +...</div></div></div></a></nav><div class="relatedPosts"><div class="headline"><i class="fas fa-thumbs-up fa-fw"></i><span>相关推荐</span></div><div class="relatedPosts-list"><a class="pagination-related" href="/2023/08/31/2023-08-31%20%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0/" title="2023-08-31 学习笔记"><img class="cover" src="https://jsd.012700.xyz/gh/jerryc127/CDN/img/material-7.png" alt="cover"><div class="info text-center"><div class="info-1"><div class="info-item-1"><i class="far fa-calendar-alt fa-fw"></i> 2023-08-31</div><div class="info-item-2">2023-08-31 学习笔记</div></div><div class="info-2"><div class="info-item-1">2023-08-31 学习笔记1. 预训练语言模型的使用问题：如何使用预训练的语言模型在已有的数据集上进行微调？ 解决方案如下： 首先下载一个预训练语言模型，这里使用的是 cardiffnlp/twitter-xlm-roberta-base-sentiment-multilingual 可以直接在Hugging Face 上下载 预处理数据集(很关键！！)，数据集如果没有处理好，模型运行过程中容易报错，例如 3. 错误解决 中就出现了因为数据集没有处理好而导致的错误 加载模型和分词器 模型训练 模型评估  代码如下： 项目配置文件 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647# hydra 配置defaults:  - _self_ # 优先级最高, 会覆盖其他配置  - override hydra/hydra_logging: disabled # 禁用hydra日志, 会覆盖其他配置  - override hydra/job_logging:...</div></div></div></a><a class="pagination-related" href="/2023/10/11/other-%E9%9A%8F%E7%AC%94-%E9%97%AE%E9%A2%98%E7%AC%94%E8%AE%B0/" title="问题笔记"><img class="cover" src="https://jsd.012700.xyz/gh/jerryc127/CDN/img/material-9.png" alt="cover"><div class="info text-center"><div class="info-1"><div class="info-item-1"><i class="far fa-calendar-alt fa-fw"></i> 2023-10-11</div><div class="info-item-2">问题笔记</div></div><div class="info-2"><div class="info-item-1">遇到的问题&amp;解决方案【持续更新~】2023-09-16 笔记解决 MP4 视频转成 m3u8 格式的问题 需求描述： 现在需要批量将一些存放在文件夹下的 MP4 视频转成 m3u8 格式，其中视频存放的位置可能是多层文件夹下，m3u8 视频需要分门别类的存放  解决思路  使用 Python + ffmpeg 实现视频的转码 使用多线程加速视频的转码   遇到的问题  多个视频不能按照预设的顺序进行转码，前一个视频还没有完成，下一个视频就中断了前一个视频的转码 本机的性能不足，需要放在服务器上运行，但是服务器上的Linux环境导致运行出现一些没有预测到的问题   最终解决方案  不使用多线程运行 使用Python...</div></div></div></a><a class="pagination-related" href="/2025/01/05/other-%E9%9A%8F%E7%AC%94-icarus-%E4%B8%BB%E9%A2%98%E5%AE%89%E8%A3%85/" title="Hexo 配置 Icarus 主题"><img class="cover" src="https://jsd.012700.xyz/gh/jerryc127/CDN/img/material-4.png" alt="cover"><div class="info text-center"><div class="info-1"><div class="info-item-1"><i class="far fa-calendar-alt fa-fw"></i> 2025-01-05</div><div class="info-item-2">Hexo 配置 Icarus 主题</div></div><div class="info-2"><div class="info-item-1">本文讲述如何给 hexo 博客网站配置。 1. 安装环境说明 首先粘贴本教程测试使用的环境，如果在安装配置中，环境有差异，请酌情修改或调整参数．  1234567891011C:\JetBrain\WebStorm\MyBlog\blog-demo-03&gt;pnpm listLegend: production dependency, optional only, dev onlyhexo-site@0.0.0 Z:\JetBrain\WebStorm\MyBlog\blog-demo-03 (PRIVATE)dependencies:@babel/preset-env 7.26.0       hexo-generator-index 3.0.0     hexo-renderer-marked 6.3.0babel-plugin-inferno 6.7.2     hexo-generator-tag 2.0.0       hexo-renderer-stylus 3.0.1hexo 7.3.0                     hexo-log 4.1.0  ...</div></div></div></a></div></div></div><div class="aside-content" id="aside-content"><div class="card-widget card-info text-center"><div class="avatar-img"><img src="/img/avatar.png" onerror="this.onerror=null;this.src='/img/loading.gif'" alt="avatar"/></div><div class="author-info-name">David</div><div class="author-info-description">Welcome to David's Blog</div><div class="site-data"><a href="/archives/"><div class="headline">文章</div><div class="length-num">27</div></a><a href="/tags/"><div class="headline">标签</div><div class="length-num">28</div></a><a href="/categories/"><div class="headline">分类</div><div class="length-num">28</div></a></div><a id="card-info-btn" target="_blank" rel="noopener external nofollow noreferrer" href="https://github.com/david-deng-01"><i class="fab fa-github"></i><span>Follow Me</span></a><div class="card-info-social-icons"><a class="social-icon" href="https://github.com/david-deng-01" rel="external nofollow noreferrer" target="_blank" title="Github"><i class="fab fa-github"></i></a><a class="social-icon" href="https://blog.csdn.net/David_0925" rel="external nofollow noreferrer" target="_blank" title="CSDN"><i class="fa fa-book-open"></i></a><a class="social-icon" href="tencent://AddContact/?fromId=45&amp;fromSubId=1&amp;subcmd=all&amp;uin=635647792&amp;website=www.oicqzone.com" rel="external nofollow noreferrer" target="_blank" title="QQ"><i class="fab fa-qq"></i></a><a class="social-icon" href="mailto:david-deng-0925@qq.com" rel="external nofollow noreferrer" target="_blank" title="Email"><i class="fas fa-envelope-open-text"></i></a></div></div><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="item-headline"><i class="fas fa-stream"></i><span>目录</span><span class="toc-percentage"></span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#%E6%B7%B1%E5%BA%A6%E5%AD%A6%E4%B9%A0%E4%BB%A3%E7%A0%81%E7%AC%94%E8%AE%B0-01"><span class="toc-number">1.</span> <span class="toc-text">深度学习代码笔记-01</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#1-%E9%85%8D%E7%BD%AE%E7%8E%AF%E5%A2%83"><span class="toc-number">1.1.</span> <span class="toc-text">1. 配置环境</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#1-1-Conda"><span class="toc-number">1.2.</span> <span class="toc-text">1.1 Conda</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#1-2-Conda-%E5%B8%B8%E7%94%A8%E5%91%BD%E4%BB%A4"><span class="toc-number">1.3.</span> <span class="toc-text">1.2 Conda 常用命令</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#1-2-%E5%AE%89%E8%A3%85Pytorch"><span class="toc-number">1.4.</span> <span class="toc-text">1.2 安装Pytorch</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#1-%E5%88%86%E8%AF%8D%E4%BB%BB%E5%8A%A1"><span class="toc-number">1.5.</span> <span class="toc-text">1. 分词任务</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E4%BB%BB%E5%8A%A1%E7%AE%80%E4%BB%8B%EF%BC%9A"><span class="toc-number">1.5.1.</span> <span class="toc-text">任务简介：</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E4%BB%BB%E5%8A%A1%E6%AD%A5%E9%AA%A4"><span class="toc-number">1.5.2.</span> <span class="toc-text">任务步骤</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Example-1-%E4%BB%A3%E7%A0%81%E8%A7%A3%E9%87%8A"><span class="toc-number">1.5.3.</span> <span class="toc-text">Example 1 代码解释</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Example2-%E4%BB%A3%E7%A0%81%E8%A7%A3%E9%87%8A"><span class="toc-number">1.5.4.</span> <span class="toc-text">Example2 代码解释</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-%E5%8F%A5%E5%AD%90%E6%83%85%E6%84%9F%E5%88%86%E7%B1%BB%E4%BB%BB%E5%8A%A1"><span class="toc-number">1.6.</span> <span class="toc-text">2. 句子情感分类任务</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#3-%E5%AF%B9%E8%AF%9D%E7%94%9F%E6%88%90%E4%BB%BB%E5%8A%A1"><span class="toc-number">1.7.</span> <span class="toc-text">3. 对话生成任务</span></a></li></ol></li></ol></div></div><div class="card-widget card-recent-post"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><div class="aside-list-item"><a class="thumbnail" href="/2025/01/05/other-%E9%9A%8F%E7%AC%94-icarus-%E4%B8%BB%E9%A2%98%E5%AE%89%E8%A3%85/" title="Hexo 配置 Icarus 主题"><img src="https://jsd.012700.xyz/gh/jerryc127/CDN/img/material-4.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Hexo 配置 Icarus 主题"/></a><div class="content"><a class="title" href="/2025/01/05/other-%E9%9A%8F%E7%AC%94-icarus-%E4%B8%BB%E9%A2%98%E5%AE%89%E8%A3%85/" title="Hexo 配置 Icarus 主题">Hexo 配置 Icarus 主题</a><time datetime="2025-01-05T08:25:00.000Z" title="发表于 2025-01-05 16:25:00">2025-01-05</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2024/03/18/other-leetcode-%E3%80%902024-03-18-LeetCode-%E5%88%B7%E9%A2%98-400%E9%81%93%E6%89%93%E5%8D%A1%E3%80%91/" title="LeetCode刷题400道打卡"><img src="https://jsd.012700.xyz/gh/jerryc127/CDN/img/material-9.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="LeetCode刷题400道打卡"/></a><div class="content"><a class="title" href="/2024/03/18/other-leetcode-%E3%80%902024-03-18-LeetCode-%E5%88%B7%E9%A2%98-400%E9%81%93%E6%89%93%E5%8D%A1%E3%80%91/" title="LeetCode刷题400道打卡">LeetCode刷题400道打卡</a><time datetime="2024-03-17T16:00:00.000Z" title="发表于 2024-03-18 00:00:00">2024-03-18</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2024/01/04/2024-01-05%20%E5%A4%9A%E6%A8%A1%E6%80%81%E6%A8%A1%E5%9E%8B%E7%BB%BC%E8%BF%B0/" title="多模态基础模型：从专家到通用助理"><img src="https://jsd.012700.xyz/gh/jerryc127/CDN/img/material-7.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="多模态基础模型：从专家到通用助理"/></a><div class="content"><a class="title" href="/2024/01/04/2024-01-05%20%E5%A4%9A%E6%A8%A1%E6%80%81%E6%A8%A1%E5%9E%8B%E7%BB%BC%E8%BF%B0/" title="多模态基础模型：从专家到通用助理">多模态基础模型：从专家到通用助理</a><time datetime="2024-01-04T14:42:08.000Z" title="发表于 2024-01-04 22:42:08">2024-01-04</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2023/12/30/%E6%A0%A1%E5%9B%AD%E7%BD%91%E8%87%AA%E5%8A%A8%E7%99%BB%E5%BD%95%E8%84%9A%E6%9C%AC/" title="校园网自动登录脚本"><img src="https://jsd.012700.xyz/gh/jerryc127/CDN/img/material-9.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="校园网自动登录脚本"/></a><div class="content"><a class="title" href="/2023/12/30/%E6%A0%A1%E5%9B%AD%E7%BD%91%E8%87%AA%E5%8A%A8%E7%99%BB%E5%BD%95%E8%84%9A%E6%9C%AC/" title="校园网自动登录脚本">校园网自动登录脚本</a><time datetime="2023-12-30T09:21:30.000Z" title="发表于 2023-12-30 17:21:30">2023-12-30</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2023/11/24/%E5%A4%9A%E6%A8%A1%E6%80%81%E8%AE%BD%E5%88%BA%E8%AF%86%E5%88%AB%E5%9F%BA%E7%BA%BF%E6%A8%A1%E5%9E%8B%E5%A4%8D%E7%8E%B0/" title="多模态讽刺识别基线模型复现"><img src="https://jsd.012700.xyz/gh/jerryc127/CDN/img/material-10.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="多模态讽刺识别基线模型复现"/></a><div class="content"><a class="title" href="/2023/11/24/%E5%A4%9A%E6%A8%A1%E6%80%81%E8%AE%BD%E5%88%BA%E8%AF%86%E5%88%AB%E5%9F%BA%E7%BA%BF%E6%A8%A1%E5%9E%8B%E5%A4%8D%E7%8E%B0/" title="多模态讽刺识别基线模型复现">多模态讽刺识别基线模型复现</a><time datetime="2023-11-24T08:51:18.000Z" title="发表于 2023-11-24 16:51:18">2023-11-24</time></div></div></div></div></div></div></main><footer id="footer"><div id="footer-wrap"><div class="copyright">&copy;2022 - 2025 By David</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener external nofollow noreferrer" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener external nofollow noreferrer" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div><div class="footer_custom_text"><a href="http://www.beian.gov.cn/portal/registerSystemInfo" rel="external nofollow noreferrer" target="_blank"> <img style="vertical-align:middle; width:20px; " src="https://cdn.jsdelivr.net/gh/David-deng-01/images/blog/icp.png"> 赣公网安备36082302000135号</a> <a href="https://beian.miit.gov.cn/" rel="external nofollow noreferrer" id="beian"  target="_blank">赣ICP备2023013705号-1</a></div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="阅读模式"><i class="fas fa-book-open"></i></button><button id="darkmode" type="button" title="日间和夜间模式切换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="单栏和双栏切换"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside-config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button class="close" id="mobile-toc-button" type="button" title="目录"><i class="fas fa-list-ul"></i></button><button id="translateLink" type="button" title="简繁转换">繁</button><button id="go-up" type="button" title="回到顶部"><span class="scroll-percent"></span><i class="fas fa-arrow-up"></i></button></div></div><div><script src="/js/utils.js"></script><script src="/js/main.js"></script><script src="/js/tw_cn.js"></script><script src="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.js"></script><div class="js-pjax"></div><script id="canvas_nest" defer="defer" color="0,0,255" opacity="0.7" zIndex="-1" count="99" mobile="false" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc/dist/canvas-nest.min.js"></script><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div></body></html>